package com.feng.sprider;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.HttpClients;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

/**
 * 获取网页中Url
 * 
 * @author feng
 *
 */
public class Get {
	
	public Get(UrlQuene urlQuene,VisitedUrlQuene visitedUrl,ImgUrlQuene imgUrl) {
		this.imgUrl = imgUrl;
		this.urlQuene = urlQuene;
		this.visitedUrl = visitedUrl;
	}

	HttpClient client = HttpClients.createDefault();

	HttpResponse response;

	ImgUrlQuene imgUrl;
	UrlQuene urlQuene ;
	VisitedUrlQuene visitedUrl;

	/**
	 * 获取网页中的Url
	 * @param url
	 */
	public void getUrls(String url ) {

		try {

			HttpGet get = new HttpGet(url);

			// 设置连接超时
			RequestConfig requestConfig = RequestConfig.custom()
					.setSocketTimeout(2000).setConnectionRequestTimeout(2000)
					.build();// 请求时间 传输时间

			get.setConfig(requestConfig);
			// 结束

			HttpResponse response = client.execute(get);

			HttpEntity he = response.getEntity();

			InputStream is = he.getContent();

			InputStreamReader isr = new InputStreamReader(is);

			BufferedReader br = new BufferedReader(isr);

			StringBuffer contents = new StringBuffer();
			String temp = null;
			// 合并到一块
			while ((temp = br.readLine()) != null) {
				contents.append(temp + "\n");
			}

			String content = contents.toString();
			/**
			 * 获取网页链接
			 */
			{
				Document document = Jsoup.parse(content);
				// 常用的为<a />标签
				Elements elements = document.getElementsByTag("a");

				for (Element es : elements) {
					String tempUrl = es.attr("href");
					// 不包含http://www.nationalgeographic.com.cn/不行
					// 不能被访问过
					if (tempUrl
							.contains("http://www.nationalgeographic.com.cn/")
							&& !visitedUrl.isExist(tempUrl)&&!urlQuene.isExist(tempUrl)) {

						urlQuene.addUrl(tempUrl);
					
					}
				}
			}

			br.close();
			isr.close();
			is.close();

		} catch (IOException e) {
			System.out.println("错误在这里");
		}
	}
	
	
	public void getImgUrls( String url){

		try {

			
			HttpGet get = new HttpGet(url);

			// 设置连接超时
			RequestConfig requestConfig = RequestConfig.custom()
					.setSocketTimeout(2000).setConnectionRequestTimeout(2000)
					.build();// 请求时间 传输时间

			get.setConfig(requestConfig);
			// 结束

			HttpResponse response = client.execute(get);

			HttpEntity he = response.getEntity();

			InputStream is = he.getContent();

			InputStreamReader isr = new InputStreamReader(is);

			BufferedReader br = new BufferedReader(isr);

			StringBuffer contents = new StringBuffer();
			String temp = null;
			// 合并到一块
			while ((temp = br.readLine()) != null) {
				contents.append(temp + "\n");
			}

			String content = contents.toString();
			/**
			 * 获取图片链接
			 */
			{
				Document document = Jsoup.parse(content);
				// 常用的为<a />标签
				Elements elements = document.getElementsByTag("img");

				for (Element es : elements) {
					String tempUrl = es.attr("src");
					// 不包含http://www.nationalgeographic.com.cn/不行
					// 不能被访问过
					
					
					if (tempUrl.contains("http://image.nationalgeographic.com.cn/")
							&& !visitedUrl.isExist(tempUrl)) {
						
						imgUrl.addUrl(tempUrl);

					}
				}
			}


			br.close();
			isr.close();
			is.close();

		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}

/*	public static void main(String[] args) {
		ImgUrlQuene imgUrl=new ImgUrlQuene ();
		UrlQuene urlQuene=new UrlQuene() ;
		VisitedUrlQuene visitedUrl =new VisitedUrlQuene();
		Get g=new Get(urlQuene, visitedUrl, imgUrl);
		
		g.getImgUrls("http://www.nationalgeographic.com.cn/");
		
	}*/
	
}
